﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace Db8.DataAccess
{
    public static class Tokenizer
    {
        internal static readonly char[] _delimiters = new char[] { ',', ' ', '\t' };
        internal static readonly string _quote = "\"";
        internal static readonly string _quotedTextPattern = "\"[^\"]*\"";

        public static List<string> Tokenize(this string data)
        {
            List<string> result = new List<string>();
            if (!string.IsNullOrWhiteSpace(data))
            {
                //result = data.Split(_delimiters).ToList();

                // find quoted text and interpret as token
                foreach (Match match in Regex.Matches(data, _quotedTextPattern))
                {
                    result.Add(match.Value.Replace(_quote, ""));
                }
                data = Regex.Replace(data, _quotedTextPattern, _delimiters[0].ToString());

                // add non-quoted tokens
                result.AddRange(data.Split(_delimiters, StringSplitOptions.RemoveEmptyEntries));
            }
            return result;
        }
    }
}
